{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example shows how to use the `gapstat` function with the `calcStats` option to generate statistics that can be used to create graphs similar to those shown in figure 1 in the original gap statistic paper (<a href=\"https://statweb.stanford.edu/~gwalther/gap\">Tibshirani 2001</a>)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gapstat import gapstat\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.datasets import make_blobs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create data set with 5 clusters\n",
    "n_samples = [600, 400, 500, 400, 550]\n",
    "centers = [[4,4],[-4,4],[0,0],[-3.25,-3.25],[4,-4]]\n",
    "cluster_std = [0.5,0.75,1.9,0.75,1.15]\n",
    "max_k = 10\n",
    "\n",
    "X,_ = make_blobs(n_samples=n_samples, centers=centers, cluster_std=cluster_std, n_features=2, random_state=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cluster the data set using the gapstat algorithm\n",
    "n_clusters, labels, stats = gapstat(X, clusterer=AgglomerativeClustering(),\n",
    "                                    max_k=max_k, calcStats=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load stats into a pandas dataframe\n",
    "stats_df = pd.DataFrame(data=stats[\"data\"], index=stats[\"index\"], columns=stats[\"columns\"])\n",
    "stats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create charts similar to those in the original gap statistic paper\n",
    "fig, axs = plt.subplots(2, 2, figsize=(10, 8))\n",
    "plt.subplots_adjust(wspace=0.5, hspace=0.5)\n",
    "\n",
    "axs[0, 0].scatter(X[:,0],X[:,1],c=labels)\n",
    "axs[0, 0].set_xlabel('x1')\n",
    "axs[0, 0].set_ylabel('x2')\n",
    "\n",
    "axs[0, 1].plot(stats_df.index, stats_df[\"W\"], marker='.', color='black', linewidth=0.5)\n",
    "axs[0, 1].set_xlabel('number of clusters k')\n",
    "axs[0, 1].set_ylabel('within sum of squares Wk')\n",
    "\n",
    "axs[1, 0].plot(stats_df.index, stats_df[\"log(W)\"], marker='$O$', color='black', markersize=8, linewidth=0.5)\n",
    "axs[1, 0].plot(stats_df.index, stats_df[\"log(W*)\"], marker='$E$', color='black', markersize=8, linewidth=0.5)\n",
    "axs[1, 0].set_xlabel('number of clusters k')\n",
    "axs[1, 0].set_ylabel('obs and exp log(Wk)')\n",
    "\n",
    "axs[1, 1].errorbar(stats_df.index, stats_df[\"Gap\"], yerr=stats_df[\"Std Err\"], capsize=3, color='black', linewidth=0.5)\n",
    "axs[1, 1].set_xlabel('number of clusters k')\n",
    "axs[1, 1].set_ylabel('Gap')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gapstat",
   "language": "python",
   "name": "gapstat"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
